﻿// author: Sylwester Sobkowicz
//
// neural network always has only one output!
// 
// In each layer exists BIAS.
// BIAS is not visible in array 'neurons', but is existing in arrays with neurons' weights

using System;
using System.Collections;
using System.Collections.Generic;

namespace NeuralNetwork
{
	class NeuralNetwork
	{
		public struct Neuron
		{
			public double[] weights;					// array of weights
			public double delta;						// error in neuron
			public double SumX, Y;						// weighted sum
		}

		private double[,] trainDataSet;					// set for training
		private double[,] testDataSet;					// set for testing

		private double beta = 1.0;						// beta rate
		private double bias = 1.0;						// bias
		private double n = 0.001;						// learning coefficient
		private double err = 0.00005;					// permissible error

		private BitArray usedVectorsInEpoch;			// using for remember which vectors has been used for training												

		private int[] typeOfActivationFunctions;		// type of activation functions for each layer (0 - sigmoid, 1 - tangent, 2 - linear

		// output values
		private double rangeBegin = -0.9;				// begin of range (hiperbolic tangent)
		private double rangeEnd = 0.9;					// end of range (hiperbolic tangent)
		private double[] outputValues;					// vector output values
		public Dictionary<double, double> values;		// values

		private int numberOfTrainingFeatures = 0;		// number of training features
		private int numberOfTrainingVectors = 0;		// number of training vectors
		private int indexColumnWithOutputs = 0;			// number of column with outputs

		private int numLayers;							// number of layers of neural network (including input layer)
		private int numInputs;							// number of neurons in input layer
		private int numHidden1;							// number of neurons in first hidden layer
		private int numHidden2;							// number of neurons in second hidden layer

		private int currentVectorIndex = 0;

		private Neuron[][] neurons;						// main array of neural network


		public NeuralNetwork(double[,] trainDataSet, double[,] testDataSet, double[] outputValues, int indexColumnWithOutputs, int activationFunctionInputLayer, int activationFunctionHidden1Layer, int activationFunctionOutputLayer, int numHidden1, int numHidden2 = 0, int activationFunctionHidden2Layer = 1)
		{
			// check out correctness of input data
			if (trainDataSet == null)
				throw new Exception("NeuralNetwork:\nSet of training data must not to be empty!");

			if (testDataSet == null)
				throw new Exception("NeuralNetwork:\nSet of testing data must not to be empty!");

			if (numHidden1 <= 0)
				throw new Exception("NeuralNetwork:\nNumber of neurons in first hidden layer must to be positive number!");

			if (numHidden2 < 0)
				throw new Exception("NeuralNetwork:\nNumber of neurons in second hidden layer must not to be negative number!");

			if (activationFunctionInputLayer != 0 && activationFunctionInputLayer != 1 && activationFunctionInputLayer != 2)
				throw new Exception("Neural Network:\nIncorrect number of activation function (input layer)!");

			if (activationFunctionHidden1Layer != 0 && activationFunctionHidden1Layer != 1 && activationFunctionHidden1Layer != 2)
				throw new Exception("Neural Network:\nIncorrect number of activation function (first hidden layer)!");

			if (activationFunctionHidden2Layer != 0 && activationFunctionHidden2Layer != 1 && activationFunctionHidden2Layer != 2)
				throw new Exception("Neural Network:\nIncorrect number of activation function (second hidden layer)!");

			if (activationFunctionOutputLayer != 0 && activationFunctionOutputLayer != 1 && activationFunctionOutputLayer != 2)
				throw new Exception("Neural Network:\nIncorrect number of activation function (output layer)!");

			if (outputValues.GetLength(0) == 0)
				throw new Exception("Neural Network:\nThere must be at least one output value!");

			// set number of hidden layers
			if (numHidden2 == 0)
				this.numLayers = 3;
			else
				this.numLayers = 4;

			// initialize basic settings
			this.trainDataSet = trainDataSet;
			this.testDataSet = testDataSet;
			this.numberOfTrainingFeatures = this.trainDataSet.GetLength(1) - 1;

			this.numInputs = this.numberOfTrainingFeatures;
			this.numHidden1 = numHidden1;
			this.numHidden2 = numHidden2;


			// activation function
			this.typeOfActivationFunctions = new int[3];
			this.typeOfActivationFunctions[0] = activationFunctionInputLayer;
			this.typeOfActivationFunctions[1] = activationFunctionHidden1Layer;

			if (numHidden2 != 0)
			{
				this.typeOfActivationFunctions[2] = activationFunctionHidden2Layer;
				this.typeOfActivationFunctions[3] = activationFunctionOutputLayer;
			}
			else
			{
				this.typeOfActivationFunctions[2] = activationFunctionOutputLayer;
			}

			// output values
			this.outputValues = outputValues;
			values = new Dictionary<double, double>();
			double range = (this.rangeEnd - this.rangeBegin) / (double)(this.outputValues.GetLength(0) - 1.0);
			
			for (int i = 0; i < this.outputValues.GetLength(0); i++)	
				values.Add((double)this.outputValues[i], this.rangeBegin + (double)i * range);

			// training data
			this.numberOfTrainingVectors = trainDataSet.GetLength(0);
			this.numberOfTrainingFeatures = trainDataSet.GetLength(1);
			this.indexColumnWithOutputs = indexColumnWithOutputs;

			// initialize BitArray
			this.usedVectorsInEpoch = new BitArray(numberOfTrainingVectors);
			this.usedVectorsInEpoch.SetAll(false);
		}


		public void CreateNetwork()
		{
			// initialize array
			this.neurons = new Neuron[this.numLayers][];


			/* input layer ----------------------------------------------------------- */

			// initialize input layer
			this.neurons[0] = new Neuron[this.numInputs];

			// initialize neurons in input layer
			for (int i = 0; i < this.numInputs; i++)
			{
				this.neurons[0][i] = new Neuron();
			}


			/* first hidden layer ---------------------------------------------------- */

			this.neurons[1] = new Neuron[this.numHidden1];

			// initialize neurons in first hidden layer
			for (int i = 0; i < this.numHidden1; i++)
			{
				this.neurons[1][i] = new Neuron();

				// set number of inputs (within bias's weight) (number of weights in this neuron)
				this.neurons[1][i].weights = new double[this.numInputs + 1];

				// set random weights to neurons
				for (int j = 0; j < this.numInputs + 1; j++)
				{
					this.neurons[1][i].weights[j] = this.RandDouble();
				}
			}


			/* second hidden layer --------------------------------------------------- */

			if (this.numHidden2 > 0)
			{
				this.neurons[2] = new Neuron[this.numHidden2];

				// initialize neurons in second hidden layer
				for (int i = 0; i < this.numHidden2; i++)
				{
					this.neurons[2][i] = new Neuron();

					// set number of inputs (within bias's weight) (number of weights in this neuron)
					this.neurons[2][i].weights = new double[this.numHidden1 + 1];

					// set random weights to neurons
					for (int j = 0; j < this.numHidden1 + 1; j++)
					{
						this.neurons[2][i].weights[j] = this.RandDouble();
					}
				}
			}


			/* output layer ---------------------------------------------------------- */

			// initialize output layer
			this.neurons[this.numLayers - 1] = new Neuron[1];

			int numberOfWeights = this.neurons[this.numLayers - 2].GetLength(0) + 1;

			// initialize neurons in output layer
			this.neurons[this.numLayers - 1][0] = new Neuron();
			
			// set number of neurons' weights
			this.neurons[this.numLayers - 1][0].weights = new double[numberOfWeights];

			// set randowm weights to neurons
			for (int i = 0; i < numberOfWeights; i++)
			{
				this.neurons[this.numLayers - 1][0].weights[i] = this.RandDouble();
			}
		}


		public void TrainNetwork()
		{
			Random rand = new Random();
			double[] vec = new double[this.numberOfTrainingFeatures];
			double previousErrorOfEpoch = 0.0, currentErrorOfEpoch = 0.0;
			int epochsCounter = 0;
			double error = 0.0;

			do
			{
				previousErrorOfEpoch = currentErrorOfEpoch;
				currentErrorOfEpoch = 0.0;
				usedVectorsInEpoch.SetAll(false);

				for (int i = 0; i < this.numberOfTrainingVectors; i++)
				{
					// random number of vector
					do
					{
						this.currentVectorIndex = rand.Next(0, this.numberOfTrainingVectors);		// range: <0, this.numberOfTrainingVectors)
					}
					while (this.usedVectorsInEpoch.Get(currentVectorIndex) != false);

					for (int j = 0; j < this.numberOfTrainingFeatures; j++)
						vec[j] = this.trainDataSet[this.currentVectorIndex, j];

					// calculate neurons' sums and Ys
					this.CalculateNeuronsSumsAndYs(vec);

					// show the output of network
					//Console.WriteLine(this.getNetworkOutput);

					// calculate errors
					currentErrorOfEpoch += this.CalculateErrors(vec);

					// correct errors
					this.ChangeWeights();

					this.usedVectorsInEpoch[this.currentVectorIndex] = true;
				}

				error = (double)(Math.Abs(currentErrorOfEpoch - previousErrorOfEpoch));

				epochsCounter++;
			}
			//while (epochsCounter < 200);
			while (epochsCounter < 20 || error > this.err);		// do it until error is enough small

			Console.WriteLine("Error: {0}", error);
			Console.WriteLine("Epochs counter: {0}", epochsCounter);
		}


		public double getNetworkOutput
		{
			get
			{
				return this.neurons[this.numLayers - 1][0].Y;
			}
		}


		private double RandDouble()
		{
			// NextDouble() give numbers from 0.0 to 1.0
			// this function give numbers from -0.1 to 0.1
			Random rand = new Random();

			return ((rand.NextDouble() / 5.0) - 0.1);
		}


		#region Activation functions and their derivatives


		private double ActivationFunction(int numberOfLayer, double sum)
		{
			if (this.typeOfActivationFunctions[numberOfLayer] == 0)
			{
				return (1.0 / (1.0 + Math.Exp((-1.0) * this.beta * sum)));
			}
			else if (this.typeOfActivationFunctions[numberOfLayer] == 1)
			{
				return ((1.0 - Math.Exp((-1.0) * this.beta * sum)) / (1.0 + Math.Exp((-1.0) * this.beta * sum)));
			}
			else if (this.typeOfActivationFunctions[numberOfLayer] == 2)
			{
				return (sum);
			}
			else
			{
				return 0.0;
			}
		}


		private double DerivativeActivationFunction(int numberOfLayer, double x)
		{
			if (this.typeOfActivationFunctions[numberOfLayer] == 0)
			{
				return ((this.beta * Math.Exp(-1.0 * this.beta * x)) / (Math.Pow((1.0 + Math.Exp(-1.0 * this.beta * x)), 2)));
			}
			else if (this.typeOfActivationFunctions[numberOfLayer] == 1)
			{
				return (((1.0 + Math.Exp(-1.0 * this.beta * x)) * (this.beta * Math.Exp(-1.0 * this.beta * x)) - (1.0 - Math.Exp(-1.0 * this.beta * x)) * (-1.0 * this.beta * Math.Exp(-1.0 * this.beta * x))) / (Math.Pow((1.0 + Math.Exp(-1.0 * this.beta * x)), 2)));
			}
			else if (this.typeOfActivationFunctions[numberOfLayer] == 2)
			{
				return 1.0;
			}
			else
			{
				return 0.0;
			}
		}


		#endregion


		#region Filling inputs, calculating SumXs and Ys in neurons


		private void FillInputLayer(double[] vec)
		{
			// function gets a line from this.trainDataSet
			for (int i = 0; i < this.numInputs; i++)
			{
				this.neurons[0][i].SumX = vec[i];
			}
		}


		// Calculate "SumX"-s in all neurons in layer
		private void CalculateSumXsInNeuronsInLayer(int numberOfLayer)
		{
			// cannot sum neuron's weights in input layer
			if (numberOfLayer != 0)
			{
				// get number of weights in neurons in this layer
				int numberOfWeights = this.neurons[numberOfLayer][0].weights.GetLength(0);

				for (int i = 0; i < this.neurons[numberOfLayer].GetLength(0); i++)
				{
					double sum = 0.0;

					// inputs * weights
					for (int j = 0; j < numberOfWeights; j++)
					{
						if (j != 0)
							sum += this.neurons[numberOfLayer - 1][j - 1].Y * this.neurons[numberOfLayer][i].weights[j];
						else
							sum += this.bias * this.neurons[numberOfLayer][i].weights[j];		// bias
					}

					this.neurons[numberOfLayer][i].SumX = sum;
				}
			}
		}


		private void CalculateNeuronsYsInLayer(int numberOfLayer)
		{
			// numberOfLayer is number of INDEX of layer (within number '0')

			for (int i = 0; i < this.neurons[numberOfLayer].GetLength(0); i++)
			{
				this.neurons[numberOfLayer][i].Y = this.ActivationFunction(numberOfLayer, this.neurons[numberOfLayer][i].SumX);
			}
		}


		#endregion


		public void CalculateNeuronsSumsAndYs(double[] vec)
		{
			// fill input layer
			this.FillInputLayer(vec);

			// Ys in input layer
			this.CalculateNeuronsYsInLayer(0);


			for (int i = 1; i < this.numLayers; i++)
			{
				// Sum in i-layer
				this.CalculateSumXsInNeuronsInLayer(i);

				// Ys in i-layer
				this.CalculateNeuronsYsInLayer(i);
			}
		}


		public double CalculateErrors(double[] vec)
		{
			// http://galaxy.agh.edu.pl/~vlsi/AI/backp_t/backprop.html

			// for each layer
			for (int i = this.numLayers - 1; i > 0; i--)
			{
				if (i != this.numLayers - 1)
				{
					//for each neuron in current layer
					for (int j = 0; j < this.neurons[i].GetLength(0); j++)
					{
						// for each neuron in (i+1) layer
						double sum = 0.0;

						for (int k = 0; k < this.neurons[i+1].GetLength(0); k++)
							sum += this.neurons[i][j].delta = this.neurons[i + 1][k].weights[j + 1];

						this.neurons[i][j].delta = sum;
					}
				}
				else
				{
					// output layer
					this.neurons[i][0].delta = this.values[vec[this.indexColumnWithOutputs]] - this.neurons[i][0].Y;
				}
			}

			return (this.neurons[this.numLayers - 1][0].delta);
		}


		public void ChangeWeights()
		{
			// http://galaxy.agh.edu.pl/~vlsi/AI/backp_t/backprop.html

			// for each layer
			for (int i = 1; i < this.numLayers; i++)
			{
				// for each neuron in current layer
				for (int j = 0; j < this.neurons[i].GetLength(0); j++)
				{
					// for each weight
					for (int k = 0; k < this.neurons[i][j].weights.GetLength(0); k++)
					{
						double y = 0.0;

						if (k != 0)
							y = this.neurons[i - 1][k - 1].Y;
						else
							y = this.bias;						// weight of bias

						this.neurons[i][j].weights[k] += this.n * this.neurons[i][j].delta * this.DerivativeActivationFunction(i, this.neurons[i][j].SumX) * y;
					}
				}
			}
		}


		public double TestVector(double[] vec)
		{
			double ans = 0.0;

			if (vec.GetLength(0) != this.numberOfTrainingFeatures)
				throw new Exception("NeuralNetwork:\nDifferent numbers of features!");

			this.CalculateNeuronsSumsAndYs(vec);

			ans = this.getNetworkOutput;

			return ans;
		}
	}
}
